# coding=utf-8
import json
import base64
import pdb
import jieba
import re
import requests
import chardet
from difflib import Differ

"""
请求post参数解码
post_data值的示例如下
{
    'mimeType': 'application/x-www-form-urlencoded; charset=UTF-8', 
    'encoding': 'base64', 
    'text': 'Y3VycmVudD0xJnJvd0NvdW50PTEwJnNlYXJjaFBocmFzZT0='
}
"""
def postDataDecode(post_data):
    param_encode = {'base64'}
    decode_data = {}
    if post_data:
        if post_data['encoding'] in param_encode:
            decode_data = base64.b64decode(post_data['text']).decode()
    return decode_data

"""
判断是否是json格式
"""
def isJson(json_str):
    try:
        json.loads(json_str)
    except ValueError:
        return False

    return True 

"""
判断是否是数字
"""
def isNumber(s):
    try:
        float(s)
        return True
    except ValueError:
        pass
 
    try:
        import unicodedata
        unicodedata.numeric(s)
        return True
    except (TypeError, ValueError):
        pass
 
    return False

"""
url拆开分析
"""
def spliUrlIntoPiece(pathStr, stop_words):
    
    tempSplit =[k for k in jieba.lcut(pathStr,cut_all=False) if k not in stop_words]
    
    return tempSplit

"""
过滤无用的字符
"""
def filterTags(html):
    if len(html) <= 0:
        return ''
    re_cdata = re.compile('//<!\[CDATA\[[^>]*//\]\]>', re.I)  # 匹配CDATA
    re_script = re.compile('<\s*script[^>]*>[^<]*<\s*/\s*script\s*>', re.I)  # Script
    #re_script = re.compile('(<|&lt;)script.*script(>|&gt;)', re.I)  # Script
    re_script = re.compile('<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>', re.I)  # Script
    re_style = re.compile('<\s*style[^>]*>[^<]*<\s*/\s*style\s*>', re.I)  # style
    re_br = re.compile('<br\s*?/?>')  # 处理换行
    re_comment = re.compile('<!--[^>]*-->')  # HTML注释
    s = re_cdata.sub('', html)  # 去掉CDATA
    s = re_script.sub('', s)  # 去掉SCRIPT
    s = re_style.sub('', s)  # 去掉style
    s = re_br.sub('', s)  # 将br转换为换行
    s = re_comment.sub('', s)  # 去掉HTML注释
    # 去掉多余的空行
    s = re.compile('\n+').sub('\n', s)
    s = re.compile('\t').sub('',s)
    s = re.compile('\r').sub('',s)
    s = re.compile('\r\n').sub('',s)
    # 去掉标签以内的空格
    #s = re.compile('[ \t]+').sub('',s)
    return s

"""
标题过滤
"""
def filterTitle(title):
    if len(title) <= 0:
        return ''
    title = title.strip().strip('.')
    title = re.compile('\n').sub('', title)
    
    return title    
"""
构造新请求头

request_headers值的示例如下
[
    {'name': 'Content-Type', 'value': 'application/x-www-form-urlencoded; charset=UTF-8'}, 
    {'name': 'Referer', 'value': 'http://www.yngp.com/bulletin.do?method=toMessagesMore'}, 
    {'name': 'User-Agent', 'value': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/602.1 (KHTML, like Gecko) splash Version/10.0 Safari/602.1'}, 
    {'name': 'Accept', 'value': '*/*'}, {'name': 'Origin', 'value': 'http://www.yngp.com'}, 
    {'name': 'X-Requested-With', 'value': 'XMLHttpRequest'}, 
    {'name': 'Content-Length', 'value': '35'}, 
    {'name': 'Cookie', 'value': 'elvasid=1c42664ffb47239b8dc96b3a991a9c51; xincaigou=49737.2912.1072.0000; JSESSIONID=feqnkvW95AdlcAGyKYHq9Uu_Eqn9rTjulx32dLWsam4H2hNuwbvU!-198674272'}
]
"""   
def structureHeader(request_headers):
    headers = {}
    if request_headers:
        for key in request_headers:
            headers[key['name']]= key['value']
        #print(headers.rstrip(',').strip())
    return headers


"""
# 递归逐字符判断字符串数据格式组成
# 如果字符串为MD5一类特殊字符串则优先识别
# 默认按照字符串数字组合进行识别
"""
def getRegFromUrlPiece(value):
    # 正则识别uuid
    regUUID = "(\\w{8}(-\\w{4}){3}-\\w{12}?)"
    # 正则识别md5
    regMd5 = "[a-fA-F0-9]{16,32}"
    regExt = "(html|shtml)"
    if(re.match(regUUID,value)):
        return regUUID
    if(re.match(regMd5,value)):
        return regMd5
    if(isNumber(value)):
        return '\d{'+str(len(value))+'}'

    if(re.match(regExt,value)):
        return '.'+value
    
    if(isinstance(value,str)):
        return '\S{'+str(len(value))+'}'
    return False
    
"""
获取不同的内容
"""

def getDifferentContent(target_html, normal_html):
    
    d = Differ()
    diff = d.compare(target_html.splitlines(), normal_html.splitlines(False))
    analyse_different_content = ''.join([val[2:] for idx,val in enumerate(diff) if val.startswith("-")]);
    different_content = analyse_different_content if analyse_different_content else ''
    
    return different_content

"""
获取a标签的href相识度结果集
url_list []
limit 页面数据列表一页多少条的最小临界值
index 重载次数
"""
def getDetailUrlStr(url_list, limit, index = 0):
    detail_url_list = []
    url_split_list = {}
    new_url_list = []
    if url_list:
        if index == 0:
            url_list_set = set(url_list)
            new_url_list = url_list
        else:
            new_url_list_set = set(url_list)    
            for url in new_url_list_set:
                new_url = ''
                url_split_list = url.lstrip('/').strip().split('/')
                stop_index = len(url_split_list)-index
                for key,value in enumerate(url_split_list):
                    if (key<stop_index) and value:
                        new_url += value+'/'
                if new_url:
                    new_url_list.append(new_url)
            url_list_set = set(new_url_list)
        if url_list_set:
            for item in url_list_set:
                count = new_url_list.count(item)
                if count > limit:
                    #print("the %s has found %d" %(item, new_url_list.count(item)))  
                    detail_url_list.append(item)
            if detail_url_list:
                return detail_url_list
        index += 1 
        if index > 3:
            return ''
        return getDetailUrlStr(url_list, limit, index)
    else:
        return ''   
"""
get请求数据
"""
def doGetRquest(url, headers, data, proxies):
    try:
        r = requests.get(url=url,headers=headers,data=data, verify=False, proxies=proxies)
        #return r.text.decode('UTF-8','ignore')
        content = r.text if r.status_code == 200 else ''
        content_type = r.headers['Content-Type'] if r.headers['Content-Type'] else ''
        return {'content':content, 'content_type':content_type}
    except Exception as e:
        print(e.args)
        print(str(e))
        print(repr(e))
        return "someting wrong!"        